home *** CD-ROM | disk | FTP | other *** search
/ Chip 2006 June / CHIP 2006-06.2.iso / program / freeware / Democracy-0.8.2.exe / xulrunner / python / downloader.py < prev    next >
Encoding:
Python Source  |  2006-04-10  |  14.1 KB  |  458 lines
  1. from database import DDBObject, defaultDatabase
  2. from threading import Thread, Event, RLock
  3. from httplib import HTTPConnection, HTTPSConnection,HTTPException
  4. from scheduler import ScheduleEvent
  5. import threadpriority
  6. import config
  7. import traceback
  8. import socket
  9. import platformutils
  10. from base64 import b64encode
  11.  
  12. from time import sleep,time
  13. from urlparse import urlparse,urljoin
  14. from os import remove, rename, access, F_OK
  15. import re
  16. import math
  17. from copy import copy
  18.  
  19. from BitTorrent import configfile
  20. from BitTorrent.download import Feedback, Multitorrent
  21. from BitTorrent.defaultargs import get_defaults
  22. from BitTorrent.parseargs import parseargs, printHelp
  23. from BitTorrent.bencode import bdecode
  24. from BitTorrent.ConvertedMetainfo import ConvertedMetainfo
  25. from BitTorrent import configfile
  26. from BitTorrent import BTFailure, CRITICAL
  27. from BitTorrent import version
  28.  
  29. import sys
  30. import os
  31. import threading
  32. from time import time, strftime
  33. from cStringIO import StringIO
  34.  
  35. from dl_daemon import daemon, command
  36.  
  37. import app
  38.  
  39. from download_utils import grabURL, parseURL, cleanFilename
  40.  
  41. defaults = get_defaults('btdownloadheadless')
  42. defaults.extend((('donated', '', ''),))
  43.  
  44. # Pass in a connection to the frontend
  45. def setDelegate(newDelegate):
  46.     global delegate
  47.     delegate = newDelegate
  48.  
  49. class DownloaderError(Exception):
  50.     pass
  51.  
  52. # Returns an HTTP auth object corresponding to the given host, path or
  53. # None if it doesn't exist
  54. def findHTTPAuth(host,path,realm = None,scheme = None):
  55.     #print "Trying to find HTTPAuth with host %s, path %s, realm %s, and scheme %s" %(host,path,realm,scheme)
  56.     ret = None
  57.     defaultDatabase.beginRead()
  58.     try:
  59.         for obj in app.globalViewList['httpauths']:
  60.             if (obj.host == host and path.startswith(obj.path) and
  61.                 (realm is None or obj.realm == realm) and
  62.                 (scheme is None or obj.authScheme == scheme)):
  63.                 ret = obj
  64.                 break
  65.     finally:
  66.         defaultDatabase.endRead()
  67.     return ret
  68.  
  69.  
  70. class HTTPAuthPassword(DDBObject):
  71.     def __init__(self,username,password,host, realm, path, authScheme="Basic"):
  72.         oldAuth = findHTTPAuth(host,path,realm,authScheme)
  73.         while not oldAuth is None:
  74.             oldAuth.remove()
  75.             oldAuth = findHTTPAuth(host,path,realm,authScheme)
  76.         self.username = username
  77.         self.password = password
  78.         self.host = host
  79.         self.realm = realm
  80.         self.path = os.path.dirname(path)
  81.         self.authScheme = authScheme
  82.         DDBObject.__init__(self)
  83.  
  84.     def getAuthToken(self):
  85.         authString = ':'
  86.         self.beginRead()
  87.         try:
  88.             authString = self.username+':'+self.password
  89.         finally:
  90.             self.endRead()
  91.         return b64encode(authString)
  92.  
  93.     def getAuthScheme(self):
  94.         ret = ""
  95.         self.beginRead()
  96.         try:
  97.             ret = self.authScheme
  98.         finally:
  99.             self.endRead()
  100.         return ret
  101.  
  102. class Downloader(DDBObject):
  103.     def __init__(self, url,item):
  104.         self.url = url
  105.         self.itemList = [item]
  106.         self.startTime = time()
  107.         self.endTime = self.startTime
  108.         self.shortFilename = self.filenameFromURL(url)
  109.         self.filename = os.path.join(config.get(config.MOVIES_DIRECTORY),'Incomplete Downloads',self.shortFilename+".part")
  110.         self.filename = self.nextFreeFilename(self.filename)
  111.         self.state = "downloading"
  112.         self.currentSize = 0
  113.         self.totalSize = -1
  114.         self.blockTimes = []
  115.         self.reasonFailed = "No Error"
  116.         self.headers = None
  117.         DDBObject.__init__(self)
  118.         self.thread = Thread(target=self.runDownloader, \
  119.                              name="downloader -- %s" % self.shortFilename)
  120.         self.thread.setDaemon(True)
  121.         self.thread.start()
  122.  
  123.     ##
  124.     # In case multiple downloaders are getting the same file, we can support multiple items
  125.     def addItem(self,item):
  126.         self.itemList.append(item)
  127.  
  128.     ##
  129.     # Returns the reason for the failure of this download
  130.     # This should only be called when the download is in the failed state
  131.     def getReasonFailed(self):
  132.         ret = ""
  133.         self.beginRead()
  134.         try:
  135.             ret = self.reasonFailed
  136.         finally:
  137.             self.endRead()
  138.         return ret
  139.  
  140.     ##
  141.     # Finds a filename that's unused and similar the the file we want
  142.     # to download
  143.     def nextFreeFilename(self, name):
  144.         if not access(name,F_OK):
  145.             return name
  146.         parts = name.split('.')
  147.         insertPoint = len(parts)-1
  148.         count = 1
  149.         parts[insertPoint:insertPoint] = [str(count)]
  150.         newname = '.'.join(parts)
  151.         while access(newname,F_OK):
  152.             count += 1
  153.             parts[insertPoint] = str(count)
  154.             newname = '.'.join(parts)
  155.         return newname
  156.  
  157.     def remove(self):
  158.         DDBObject.remove(self)
  159.  
  160.     ##
  161.     # Returns the URL we're downloading
  162.     def getURL(self):
  163.         self.beginRead()
  164.         ret = self.url
  165.         self.endRead()
  166.         return ret
  167.     ##    
  168.     # Returns the state of the download: downloading, paused, stopped,
  169.     # failed, or finished
  170.     def getState(self):
  171.         self.beginRead()
  172.         ret = self.state
  173.         self.endRead()
  174.         return ret
  175.  
  176.     ##
  177.     # Returns the total size of the download in bytes
  178.     def getTotalSize(self):
  179.         self.beginRead()
  180.         ret = self.totalSize
  181.         self.endRead()
  182.         return ret
  183.  
  184.     ##
  185.     # Returns the current amount downloaded in bytes
  186.     def getCurrentSize(self):
  187.         self.beginRead()
  188.         ret = self.currentSize
  189.         self.endRead()
  190.         return ret
  191.  
  192.     ##
  193.     # Returns a float with the estimated number of seconds left
  194.     def getETA(self):
  195.         self.beginRead()
  196.         try:
  197.             rate = self.getRate()
  198.             if rate != 0:
  199.                 eta = (self.totalSize - self.currentSize)/rate
  200.                 if eta < 0:
  201.                     eta = 0
  202.             else:
  203.                 eta = 0
  204.         finally:
  205.             self.endRead()
  206.         return eta
  207.  
  208.     ##
  209.     # Returns a float with the download rate in bytes per second
  210.     def getRate(self):
  211.         now = time()
  212.         self.beginRead()
  213.         try:
  214.             if self.endTime != self.startTime:
  215.                 rate = self.currentSize/(self.endTime-self.startTime)
  216.             else:
  217.                 try:
  218.                     if (now-self.blockTimes[0][0]) != 0:
  219.                         rate=(self.blockTimes[-1][1]-self.blockTimes[0][1])/(now-self.blockTimes[0][0])
  220.                     else:
  221.                         rate = 0
  222.                 except IndexError:
  223.                     rate = 0
  224.         finally:
  225.             self.endRead()
  226.         return rate
  227.  
  228.     ##
  229.     # Returns the filename that we're downloading to. Should not be
  230.     # called until state is "finished."
  231.     def getFilename(self):
  232.         self.beginRead()
  233.         ret = self.filename
  234.         self.endRead()
  235.         return ret
  236.  
  237.     ##
  238.     # Returns a reasonable filename for saving the given url
  239.     def filenameFromURL(self,url):
  240.         (scheme, host, path, params, query, fragment) = parseURL(url)
  241.         if len(path):
  242.             try:
  243.                 ret = re.compile("^.*?([^/]+)/?$").search(path).expand("\\1")
  244.                 return cleanFilename(ret)
  245.  
  246.             except:
  247.                 return 'unknown'
  248.         else:
  249.             return "unknown"
  250.  
  251.     def runDownloader(self, retry = False):
  252.         pass
  253.  
  254.     ##
  255.     # Called by pickle during serialization
  256.     def __getstate__(self):
  257.         temp = copy(self.__dict__)
  258.         temp["thread"] = None
  259.         return (0,temp)
  260.  
  261.     ##
  262.     # Called by pickle during deserialization
  263.     def __setstate__(self,state):
  264.         (version, data) = state
  265.         assert(version == 0)
  266.         self.__dict__ = data
  267.         self.filename = config.ensureMigratedMoviePath(self.filename)
  268.         if self.getState() == "downloading":
  269.             ScheduleEvent(0, lambda :self.runDownloader(retry = True),False)
  270.  
  271. # Download an item using our separate download process
  272. # Pass in url, item, and contentType to create
  273. # Pass in localDownloadData to create from data found in local downloader
  274. class RemoteDownloader(Downloader):
  275.     def __init__(self, url = None,item = None,contentType = None,
  276.                  localDownloadData = None):
  277.         if localDownloadData is None:
  278.             self.dlid = "noid"
  279.             self.contentType = contentType
  280.             self.eta = 0
  281.             self.rate = 0
  282.             Downloader.__init__(self,url,item)
  283.         else:
  284.             self.__dict__ = localDownloadData
  285.             self.dlid = 'noid'
  286.             self.eta = 0
  287.             self.rate = 0
  288.             if self.dlerType == 'BitTorrent':
  289.                 self.contentType = 'application/x-bittorrent'
  290.             else:
  291.                 self.contentType = 'video/x-unknown'
  292.             self.thread = Thread(target=self.restoreLocalDownload,
  293.                     name="restoring old downloader -- %s" % self.shortFilename)
  294.             self.thread.setDaemon(True)
  295.             self.thread.start()
  296.             
  297.     @classmethod
  298.     def initializeDaemon(cls):
  299.         RemoteDownloader.dldaemon = daemon.ControllerDaemon()
  300.  
  301.     @classmethod
  302.     def updateStatus(cls, data):
  303.         view = app.globalViewList['remoteDownloads'].filterWithIndex(
  304.             app.globalIndexList['downloadsByDLID'],data['dlid'])
  305.         try:
  306.             view.resetCursor()
  307.             self = view.getNext()
  308.         finally:   
  309.             app.globalViewList['remoteDownloads'].removeView(view)
  310.         if not self is None:
  311.             oldState = self.state
  312.             for key in data.keys():
  313.                 self.__dict__[key] = data[key]
  314.             # Store the time the download finished
  315.             if ((self.state in ['finished','uploading']) and
  316.                 (oldState not in ['finished', 'uploading'])):
  317.                 for item in self.itemList:
  318.                     item.setDownloadedTime()
  319.             for item in self.itemList:
  320.                 item.beginChange()
  321.                 item.endChange()
  322.  
  323.     def restoreLocalDownload(self):
  324.         """Restore a previously running download."""
  325.         c = command.GenerateDownloadID(RemoteDownloader.dldaemon)
  326.         self.dlid = c.send()
  327.         restoreData = self.__dict__.copy()
  328.         del restoreData['thread']
  329.         del restoreData['itemList']
  330.         c = command.RestoreDownloaderCommand(RemoteDownloader.dldaemon,
  331.                 restoreData)
  332.         c.send()
  333.         #FIXME: This is sooo slow...
  334.         app.globalViewList['remoteDownloads'].recomputeIndex(app.globalIndexList['downloadsByDLID'])
  335.         
  336.     ##
  337.     # This is the actual download thread.
  338.     def runDownloader(self, retry = False):
  339.         if not retry:
  340.             c = command.StartNewDownloadCommand(RemoteDownloader.dldaemon,
  341.                                                 self.url, self.contentType)
  342.             self.dlid = c.send()
  343.             #FIXME: This is sooo slow...
  344.             app.globalViewList['remoteDownloads'].recomputeIndex(app.globalIndexList['downloadsByDLID'])
  345.  
  346.     ##
  347.     # Pauses the download.
  348.     def pause(self):
  349.         c = command.PauseDownloadCommand(RemoteDownloader.dldaemon,
  350.                                             self.dlid)
  351.         c.send(block=False)
  352.  
  353.     ##
  354.     # Stops the download and removes the partially downloaded
  355.     # file.
  356.     def stop(self):
  357.         c = command.StopDownloadCommand(RemoteDownloader.dldaemon,
  358.                                             self.dlid)
  359.         c.send(block=False)
  360.  
  361.     ##
  362.     # Continues a paused or stopped download thread
  363.     def start(self):
  364.         c = command.StartDownloadCommand(RemoteDownloader.dldaemon,
  365.                                             self.dlid)
  366.         c.send(block=False)
  367.  
  368.     def getRate(self):
  369.         return self.rate
  370.  
  371.     def getETA(self):
  372.         return self.eta
  373.  
  374.     ##
  375.     # Removes downloader from the database
  376.     def remove(self):
  377.         self.stop()
  378.         Downloader.remove(self)
  379.  
  380.     ##
  381.     # Called by pickle during serialization
  382.     def __getstate__(self):
  383.         temp = copy(self.__dict__)
  384.         temp["thread"] = None
  385.         return (0,temp)
  386.  
  387.     ##
  388.     # Called by pickle during deserialization
  389.     def __setstate__(self,state):
  390.         (version, data) = state
  391.         self.__dict__ = copy(data)
  392.         if data['dlid'] == 'noid':
  393.             self.thread = Thread(target=self.runDownloader, \
  394.                                  name="downloader -- %s" % self.shortFilename)
  395.             self.thread.setDaemon(True)
  396.             self.thread.start()
  397.         elif data['state'] in ['downloading','uploading']:
  398.             del data['itemList']
  399.             c = command.RestoreDownloaderCommand(RemoteDownloader.dldaemon, data)
  400.             c.send(retry = True, block = False)
  401.  
  402. ##
  403. # For upgrading from old versions of the database
  404. class HTTPDownloader(Downloader):
  405.     pass
  406.  
  407. ##
  408. # For upgrading from old versions of the database
  409. class BTDisplay:
  410.     def __setstate__(self,state):
  411.         (version, data) = state
  412.         self.__dict__ = data
  413.  
  414. ##
  415. # For upgrading from old versions of the database
  416. class BTDownloader(Downloader):
  417.     pass
  418.  
  419. ##
  420. # Kill the main BitTorrent thread
  421. #
  422. # This should be called before closing the app
  423. def shutdownDownloader():
  424.     c = command.ShutDownCommand(RemoteDownloader.dldaemon)
  425.     c.send()    
  426.  
  427. class DownloaderFactory:
  428.     lock = RLock()
  429.     def __init__(self,item):
  430.         self.item = item
  431.  
  432.     def getDownloader(self,url):
  433.         info = grabURL(url,'GET')
  434.         if info is None:
  435.             return None
  436.         # FIXME: uncomment these 2 lines and comment the 3 above to
  437.         # enable the download daemon
  438.  
  439.         else:
  440.             return RemoteDownloader(info['updated-url'],self.item, info['content-type'])
  441.  
  442. if __name__ == "__main__":
  443.     def printsaved():
  444.         print "Saved!"
  445.     def displayDLStatus(dler):
  446.         print dler.getState()
  447.         print str(dler.getCurrentSize()) + " of " + str(dler.getTotalSize())
  448.         print str(dler.getETA()) + " seconds remaining"
  449.         print str(dler.getRate()) + " bytes/sec"
  450.         print "Saving to " + dler.getFilename()
  451.     factory = DownloaderFactory(DDBObject())
  452.     x = factory.getDownloader("http://www.blogtorrent.com/demo/btdownload.php?type=torrent&file=SatisfactionWeb.mov.torrent")
  453.     y = factory.getDownloader("http://www.vimeo.com/clips/2005/04/05/vimeo.thelastminute.613.mov")
  454.     ScheduleEvent(2,lambda :displayDLStatus(x),True)
  455.     ScheduleEvent(2,lambda :displayDLStatus(y),True)
  456.     sleep(60)
  457.     x.stop()
  458.